import sys
import re

from google.appengine.api import urlfetch

charset = 'gbk'
title = "not found"

def getTags(input, tag):
    lowerThanCharCounter = 0
    lowerThanCharPos = 0
    i = 0
    tagPos = []
    taglist = []
    
    size = len(input)
    for i in range(0, size): 
        convertedCharacter = input[i]
        if (lowerThanCharCounter > 0):
            if(convertedCharacter == '>'):
                lowerThanCharCounter = lowerThanCharCounter - 1
                biaoqian = input[lowerThanCharPos:i+1]
                if (biaoqian.startswith('<'+tag) or biaoqian.startswith('<'+tag.upper())):
                    tagPos.append(lowerThanCharPos)
                elif (biaoqian.startswith('</'+tag) or biaoqian.startswith('</'+tag.upper())):
                    if (tagPos.__len__() < 1):
                        continue
                    tempTagPos = tagPos.pop()
                    strTag = input[tempTagPos:i+1]
                    taglist.append(strTag)
            
        if (convertedCharacter == '<'):
            lowerThanCharCounter = lowerThanCharCounter + 1
            lowerThanCharPos = i  
    return taglist

def getCharacterCount(str):
    char_reg = re.compile('[\x80-\xff]')
    return len(char_reg.findall(str)) / 2

def getHrefCharRate(str):
    char_count = 0.0
    href_count = 0.0
    href_list = []
    char_count = getCharacterCount(str)
    if (char_count == 0):
        return 0
    href_list = getTags(str, 'a')
    for s in href_list:
        href_count += getCharacterCount(s)
    rate = href_count / char_count
    return rate

def compareTwoStrings(s1, s2):
    rate1 = getHrefCharRate(s1)
    rate2 = getHrefCharRate(s2)
    if (rate1 < rate2):
        return -1
    elif (rate1 == rate2):
        return 0
    else:
        return 1

def getDataFromUrl(strUrl):
    global charset
    global title
    charset = 'gbk'
    regex_charset = "(?i)<meta[\\s\\S]+?charset=(.+?)[\"\\s][\\s\\S]*?>"
    regex_title = "(?i)<[\\s]*?title[^>]*?>([\\s\\S]*?)<[\\s]*?\\/[\\s]*?title[\\s]*?>"
    data = urlfetch.fetch(strUrl).content
    matcher = re.search(regex_charset, data)
    if matcher:
        if (matcher.group(1) == 'utf-8' or matcher.group(1) == 'UTF-8'):
            #data = data.decode('utf-8')
            charset = 'utf-8'
        elif (matcher.group(1) == 'gb2312' or matcher.group(1) == 'GB2312'):
            charset = 'gb2312'
    else:
         print 'not match!'
    matcher = re.search(regex_title, data)
    if matcher:
        title = '<head><center>' + matcher.group(1) + '</center></head>'
    else:
        title = "no title!"
    return data

def getMainContent(input):
  #  input = urlfetch.fetch(url).content
  #  input.decode('utf-8').encode('gbk')
    list_table = getTags(input, 'table')
    list_div = getTags(input, 'div')
    result_table = extractContent(list_table)
    result_div = extractContent(list_div)
    if (len(result_table) > len(result_div)):
        result = result_table
    else:
        result = result_div
        
   # result = refine2(result)
    return result
   

def extractContent(list):
    regex_p = "(?i)<p[^<]*>"
    regex_br = "(?i)<br[^<]*>"
    regex_script = "(?i)<[\\s]*?script[^>]*?>[\\s\\S]*?<[\\s]*?\\/[\\s]*?script[\\s]*?>"
    regex_style = "(?i)<[\\s]*?style[^>]*?>[\\s\\S]*?<[\\s]*?\\/[\\s]*?style[\\s]*?>"
    regex_html = "<[^>]+>"
    regex_comment = "<!--[\\s\\S]*?-->"
    regex_custom_1 = "&#?[\\w]*;"
  #  regex_object = "(?i)<[\\s]*?object[^>]*?>[\\s\\S]*?<[\\s]*?\\/[\\s]*?object[\\s]*?>"
    input = ''

    needToRemove = []
    for s in list:
        count = getCharacterCount(s)
        rate = getHrefCharRate(s)
        if (count < 200 or rate > 0.5):
            needToRemove.append(s)
    
    for s in needToRemove:
        list.remove(s)
    
    if (len(list) < 1):
        return "empty!"
    
    list.sort(compareTwoStrings)
    input = list[0]
   # input = refine1(input)
    input = re.sub(regex_p, '[p]', input)
    input = re.sub(regex_br, '[br]', input)
    input = re.sub(regex_script, '', input)
    input = re.sub(regex_style, '', input)
    input = re.sub(regex_comment, '', input)
    input = re.sub(regex_html, '', input)
    input = re.sub(regex_custom_1, '', input)
    input = re.sub('\\[p]', '<p>', input)
    input = re.sub('\\[br]', '<br>', input)
  
    return input

def refine1(input):
    split = re.split('\r\n', input)
    for s in split:
        if (getHrefCharRate(s) > 0.5):
            input = re.sub(s, '\r\n', input)
    return input

def refine2(input):
    result = ''
    blank_reg = re.compile('\\s')
    split = re.split('\n', input)
    for s in split:
        if (len(blank_reg.findall(s)) == len(s)):
            continue
        else:
            result += '<p>' + s + '\n'
    return result

def getTitle():
    global title
    return title

def getCharset():
    global charset
    return charset   
